<!DOCTYPE html>
<html lang=zh>
<head><meta name="generator" content="Hexo 3.9.0">
    <!-- so meta -->
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="HandheldFriendly" content="True">
    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
    <meta name="description" content="前言之前介绍了如何写一个最简单的爬虫，今天我们来讲一讲 scrapy 的选择器。 在爬虫程序的 parse 方法里，response 对象承载着爬虫爬取得到的最原始的数据，如何从这些混乱的数据中得到我们想要的数据就是接下来我们应该思考的。其实 python 爬虫和 nodejs 爬虫在这方面很类似，都是将原始数据传递给一个解析器，然后通过 xml 或者 html 的选择器去获取数据。只是不同的是，">
<meta name="keywords" content="毕设,python">
<meta property="og:type" content="article">
<meta property="og:title" content="Python scrapy 选择器">
<meta property="og:url" content="https://andyliwr.github.io/2017/10/28/python_scrapy_selector/index.html">
<meta property="og:site_name" content="月光倾城的代码小屋">
<meta property="og:description" content="前言之前介绍了如何写一个最简单的爬虫，今天我们来讲一讲 scrapy 的选择器。 在爬虫程序的 parse 方法里，response 对象承载着爬虫爬取得到的最原始的数据，如何从这些混乱的数据中得到我们想要的数据就是接下来我们应该思考的。其实 python 爬虫和 nodejs 爬虫在这方面很类似，都是将原始数据传递给一个解析器，然后通过 xml 或者 html 的选择器去获取数据。只是不同的是，">
<meta property="og:locale" content="zh-CN">
<meta property="og:image" content="http://ouizhbgin.bkt.clouddn.com/blog/2017/10/28/scrapy_shell.png">
<meta property="og:updated_time" content="2020-01-15T06:30:09.120Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Python scrapy 选择器">
<meta name="twitter:description" content="前言之前介绍了如何写一个最简单的爬虫，今天我们来讲一讲 scrapy 的选择器。 在爬虫程序的 parse 方法里，response 对象承载着爬虫爬取得到的最原始的数据，如何从这些混乱的数据中得到我们想要的数据就是接下来我们应该思考的。其实 python 爬虫和 nodejs 爬虫在这方面很类似，都是将原始数据传递给一个解析器，然后通过 xml 或者 html 的选择器去获取数据。只是不同的是，">
<meta name="twitter:image" content="http://ouizhbgin.bkt.clouddn.com/blog/2017/10/28/scrapy_shell.png">
    
    
        
          
              <link rel="shortcut icon" href="/images/favicon.ico">
          
        
        
          
            <link rel="icon" type="image/png" href="/images/favicon-192x192.png" sizes="192x192">
          
        
        
          
            <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon.png">
          
        
    
    <!-- title -->
    <title>Python scrapy 选择器</title>
    <!-- styles -->
    <link rel="stylesheet" href="/css/style.css">
    <!-- persian styles -->
    
      <link rel="stylesheet" href="/css/rtl.css">
    
    <!-- rss -->
    
    
</head>

<body class="max-width mx-auto px3 ltr">
    
      <div id="header-post">
  <a id="menu-icon" href="#"><i class="fas fa-bars fa-lg"></i></a>
  <a id="menu-icon-tablet" href="#"><i class="fas fa-bars fa-lg"></i></a>
  <a id="top-icon-tablet" href="#" onclick="$('html, body').animate({ scrollTop: 0 }, 'fast');" style="display:none;"><i class="fas fa-chevron-up fa-lg"></i></a>
  <span id="menu">
    <span id="nav">
      <ul>
         
          <li><a href="/">首页</a></li>
         
          <li><a href="/archives/">归档</a></li>
         
          <li><a href="/search/">搜索</a></li>
         
          <li><a href="/tags/">标签</a></li>
         
          <li><a href="/others/">其他文章</a></li>
        
      </ul>
    </span>
    <br/>
    <span id="actions">
      <ul>
        
        <li><a class="icon" href="/2017/10/30/koa2_await_suport/"><i class="fas fa-chevron-left" aria-hidden="true" onmouseover="$('#i-prev').toggle();" onmouseout="$('#i-prev').toggle();"></i></a></li>
        
        
        <li><a class="icon" href="/2017/10/28/python_scrapy_rumen/"><i class="fas fa-chevron-right" aria-hidden="true" onmouseover="$('#i-next').toggle();" onmouseout="$('#i-next').toggle();"></i></a></li>
        
        <li><a class="icon" href="#" onclick="$('html, body').animate({ scrollTop: 0 }, 'fast');"><i class="fas fa-chevron-up" aria-hidden="true" onmouseover="$('#i-top').toggle();" onmouseout="$('#i-top').toggle();"></i></a></li>
        <li><a class="icon" href="#"><i class="fas fa-share-alt" aria-hidden="true" onmouseover="$('#i-share').toggle();" onmouseout="$('#i-share').toggle();" onclick="$('#share').toggle();return false;"></i></a></li>
      </ul>
      <span id="i-prev" class="info" style="display:none;">上一篇</span>
      <span id="i-next" class="info" style="display:none;">下一篇</span>
      <span id="i-top" class="info" style="display:none;">返回顶部</span>
      <span id="i-share" class="info" style="display:none;">分享文章</span>
    </span>
    <br/>
    <div id="share" style="display: none">
      <ul>
  <li><a class="icon" href="http://www.facebook.com/sharer.php?u=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/"><i class="fab fa-facebook " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://twitter.com/share?url=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/&text=Python scrapy 选择器"><i class="fab fa-twitter " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.linkedin.com/shareArticle?url=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/&title=Python scrapy 选择器"><i class="fab fa-linkedin " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://pinterest.com/pin/create/bookmarklet/?url=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/&is_video=false&description=Python scrapy 选择器"><i class="fab fa-pinterest " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="mailto:?subject=Python scrapy 选择器&body=Check out this article: https://andyliwr.github.io/2017/10/28/python_scrapy_selector/"><i class="fas fa-envelope " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://getpocket.com/save?url=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/&title=Python scrapy 选择器"><i class="fab fa-get-pocket " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://reddit.com/submit?url=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/&title=Python scrapy 选择器"><i class="fab fa-reddit " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.stumbleupon.com/submit?url=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/&title=Python scrapy 选择器"><i class="fab fa-stumbleupon " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://digg.com/submit?url=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/&title=Python scrapy 选择器"><i class="fab fa-digg " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.tumblr.com/share/link?url=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/&name=Python scrapy 选择器&description="><i class="fab fa-tumblr " aria-hidden="true"></i></a></li>
</ul>

    </div>
    <div id="toc">
      <ol class="toc"><li class="toc-item toc-level-4"><a class="toc-link" href="#前言"><span class="toc-number">1.</span> <span class="toc-text">前言</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#选择器"><span class="toc-number">2.</span> <span class="toc-text">选择器</span></a></li></ol>
    </div>
  </span>
</div>

    
    <div class="content index py4">
        
        <article class="post" itemscope itemtype="http://schema.org/BlogPosting">
  <header>
    
    <h1 class="posttitle" itemprop="name headline">
        Python scrapy 选择器
    </h1>



    <div class="meta">
      <span class="author" itemprop="author" itemscope itemtype="http://schema.org/Person">
        <span itemprop="name">月光倾城的代码小屋</span>
      </span>
      
    <div class="postdate">
        <time datetime="2017-10-28T13:17:00.000Z" itemprop="datePublished">2017-10-28</time>
    </div>


      

      
    <div class="article-tag">
        <i class="fas fa-tag"></i>
        <a class="tag-link" href="/tags/python/">python</a>, <a class="tag-link" href="/tags/毕设/">毕设</a>
    </div>


    </div>
  </header>
  

  <div class="content" itemprop="articleBody">
    <h4 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h4><p>之前介绍了如何写一个最简单的爬虫，今天我们来讲一讲 scrapy 的选择器。 在爬虫程序的 parse 方法里，response 对象承载着爬虫爬取得到的最原始的数据，如何从这些混乱的数据中得到我们想要的数据就是接下来我们应该思考的。其实 python 爬虫和 nodejs 爬虫在这方面很类似，都是将原始数据传递给一个解析器，然后通过 xml 或者 html 的选择器去获取数据。只是不同的是，nodejs 是使用的是 cheerio 而 python 使用的是 xpath 或者 css。</p>
<p>再介绍选择器之前介绍一个很实用的工具，它是 scrapy 自带的 shell 工具，通过输入<code>scrapy shell &lt;url&gt;</code>可以进入命令行模式，这样可以让你在不写爬虫程序的情况下测试你的选择器，例如：<br><img src="http://ouizhbgin.bkt.clouddn.com/blog/2017/10/28/scrapy_shell.png" alt="scrapy shell"></p>
<h4 id="选择器"><a href="#选择器" class="headerlink" title="选择器"></a>选择器</h4><p>scrapy 提供两种选择器 — reponse.css()和 response.xpath()，作为前端我选择 css，因为熟悉。</p>
<p><strong>1、css 选择器</strong></p>
<ul>
<li><strong>获取页面标题</strong></li>
</ul>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">response.css(<span class="string">'title::text'</span>).extract()</span><br></pre></td></tr></table></figure>
<p>通过<code>response.css(&#39;title&#39;)</code>获取到的只是一个<code>&lt;title&gt;&lt;/title&gt;</code>元素，只有在后面加上<code>::text</code>，才会得到元素里面具体的文字。还没完，通过<code>response.css(&#39;title::text&#39;)</code>获取到的只是一个 Selector 对象，只有调用<code>.extract()</code>才会的得到真正的内容。如果数据有多个但是你只想要第一个的话，那就使用<code>.extract_first()</code></p>
<ul>
<li><strong>获取元素的属性</strong></li>
</ul>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">response.css(<span class="string">'.result-item-title&gt;a::attr(href)'</span>).extract()</span><br></pre></td></tr></table></figure>
<p>通过<code>::attr(属性名)</code>就能获取到这个元素的属性值，抓取 a 标签里的 href 属性或者获取 img 元素的图片地址都需要这么干。</p>
<ul>
<li><strong>去掉获取到的文字中的空格</strong><br>使用 python 字符串对象的 strip 方法就能轻易的做到这一点，不过有时候获取的数据有多条的时候，你得写一个循环去帮你逐条处理。</li>
</ul>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">item[<span class="string">'author'</span>] = response.css(<span class="string">'.result-game-item-info .result-game-item-info-tag:nth-child(1)&gt;a::text'</span>).extract()</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">0</span>, len(item[<span class="string">'author'</span>])):</span><br><span class="line">    item[<span class="string">'author'</span>][i] = item[<span class="string">'author'</span>][i].strip()</span><br></pre></td></tr></table></figure>
<pre><code>去掉换行的话，使用`.replace(&apos;\n&apos;, &apos;&apos;)`就可以了。
</code></pre><ul>
<li><strong>进阶用法—使用 re 和 set</strong><br>re 可以提取数据的公共部分。我们假设一种情形，你要获取的 html 是这样子的：</li>
</ul>
<figure class="highlight angelscript"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">&lt;p <span class="keyword">class</span>="<span class="symbol">target</span>"&gt;</span><br><span class="line">  哈哈哈</span><br><span class="line">  &lt;<span class="symbol">em</span>&gt;<span class="symbol">hahah</span>&lt;/<span class="symbol">em</span>&gt;</span><br><span class="line">  哈哈哈</span><br><span class="line">&lt;/<span class="symbol">p</span>&gt;</span><br></pre></td></tr></table></figure>
<pre><code>通过`response.css(&apos;.target::text()&apos;).extract()`得到的只是`哈哈哈&lt;em&gt;hahah&lt;/em&gt;哈哈哈`，但是你想要的其实是“哈哈哈hahah哈哈哈”，这个时候可以使用re方法来过滤掉`&lt;em&gt;&lt;/em&gt;`，使用方法如下：
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">response.css(<span class="string">'.target'</span>).extract()</span><br><span class="line"><span class="comment"># 输出</span></span><br><span class="line">&lt;p class="target"&gt;哈哈哈&lt;em&gt;hahah&lt;/em&gt;哈哈哈&lt;/p&gt;</span><br><span class="line"></span><br><span class="line">response.css(<span class="string">'.target:text'</span>).extract()</span><br><span class="line"><span class="comment"># 输出</span></span><br><span class="line">哈哈哈 \n 哈哈哈</span><br><span class="line"></span><br><span class="line">response.css(<span class="string">'.target'</span>).extract().re(<span class="string">r'(.*)&lt;em&gt;(.*)&lt;/em&gt;(.*)'</span>)</span><br><span class="line"><span class="comment"># 输出</span></span><br><span class="line">哈哈哈hahah哈哈哈</span><br></pre></td></tr></table></figure>
</code></pre><p>set 方法的用法，我也研究得不是很清楚，可以参考官方给的<a href="https://docs.scrapy.org/en/latest/topics/selectors.html#set-operations" target="_blank" rel="noopener">实例</a></p>
<p><strong>2、xpath 选择器</strong><br>这里只能给个<a href="https://docs.scrapy.org/en/latest/topics/selectors.html" target="_blank" rel="noopener">传送门</a>了，大致的原理都是一样的，区别就是 class 和 id 选择器，属性选择器的写法有所不同。</p>
<p>在接下来的文章，我将继续介绍<code>scrapy</code>的更多内容，敬请期待~</p>

  </div>
  <div class="statement">
    <hr>  
    1. 如果本文帮到了您，不妨点一下右上角的”<i class="fas fa-share-alt"></i>“按钮。<br>
    2. 除非注明，本博文章均为原创，转载请以链接形式标明本文地址。<br>
    3. 如果你有任何疑问，欢迎发送邮件到我的邮箱<a href="mailto:andyliwr@outlook.com">andyliwr@outlook.com<br>
  </div>
</article>


    <div class="blog-post-comments">
        <div id="gitalk_thread">
            <noscript>加载评论需要在浏览器启用 JavaScript 脚本支持。</noscript>
        </div>
    </div>



        
          <div id="footer-post-container">
  <div id="footer-post">

    <div id="nav-footer" style="display: none">
      <ul>
         
          <li><a href="/">首页</a></li>
         
          <li><a href="/archives/">归档</a></li>
         
          <li><a href="/search/">搜索</a></li>
         
          <li><a href="/tags/">标签</a></li>
         
          <li><a href="/others/">其他文章</a></li>
        
      </ul>
    </div>

    <div id="toc-footer" style="display: none">
      <ol class="toc"><li class="toc-item toc-level-4"><a class="toc-link" href="#前言"><span class="toc-number">1.</span> <span class="toc-text">前言</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#选择器"><span class="toc-number">2.</span> <span class="toc-text">选择器</span></a></li></ol>
    </div>

    <div id="share-footer" style="display: none">
      <ul>
  <li><a class="icon" href="http://www.facebook.com/sharer.php?u=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/"><i class="fab fa-facebook fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://twitter.com/share?url=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/&text=Python scrapy 选择器"><i class="fab fa-twitter fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.linkedin.com/shareArticle?url=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/&title=Python scrapy 选择器"><i class="fab fa-linkedin fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://pinterest.com/pin/create/bookmarklet/?url=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/&is_video=false&description=Python scrapy 选择器"><i class="fab fa-pinterest fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="mailto:?subject=Python scrapy 选择器&body=Check out this article: https://andyliwr.github.io/2017/10/28/python_scrapy_selector/"><i class="fas fa-envelope fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://getpocket.com/save?url=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/&title=Python scrapy 选择器"><i class="fab fa-get-pocket fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://reddit.com/submit?url=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/&title=Python scrapy 选择器"><i class="fab fa-reddit fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.stumbleupon.com/submit?url=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/&title=Python scrapy 选择器"><i class="fab fa-stumbleupon fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://digg.com/submit?url=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/&title=Python scrapy 选择器"><i class="fab fa-digg fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.tumblr.com/share/link?url=https://andyliwr.github.io/2017/10/28/python_scrapy_selector/&name=Python scrapy 选择器&description="><i class="fab fa-tumblr fa-lg" aria-hidden="true"></i></a></li>
</ul>

    </div>

    <div id="actions-footer">
        <a id="menu" class="icon" href="#" onclick="$('#nav-footer').toggle();return false;"><i class="fas fa-bars fa-lg" aria-hidden="true"></i> 菜单</a>
        <a id="toc" class="icon" href="#" onclick="$('#toc-footer').toggle();return false;"><i class="fas fa-list fa-lg" aria-hidden="true"></i> 目录</a>
        <a id="share" class="icon" href="#" onclick="$('#share-footer').toggle();return false;"><i class="fas fa-share-alt fa-lg" aria-hidden="true"></i> 分享</a>
        <a id="top" style="display:none" class="icon" href="#" onclick="$('html, body').animate({ scrollTop: 0 }, 'fast');"><i class="fas fa-chevron-up fa-lg" aria-hidden="true"></i> 返回顶部</a>
    </div>

  </div>
</div>

        
        <footer id="footer">
  <div class="footer-left">
    Copyright &copy; 2020 Dikang Li
  </div>
  <div class="footer-right">
    <nav>
      <ul>
         
          <li><a href="/">首页</a></li>
         
          <li><a href="/archives/">归档</a></li>
         
          <li><a href="/search/">搜索</a></li>
         
          <li><a href="/tags/">标签</a></li>
         
          <li><a href="/others/">其他文章</a></li>
        
      </ul>
    </nav>
  </div>
</footer>

    </div>
</body>
</html>
<!-- styles -->
<link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
<link rel="stylesheet" href="/lib/justified-gallery/css/justifiedGallery.min.css">
<link rel="stylesheet" href="/lib/fancybox/jquery.fancybox.css">

<link rel="stylesheet" href="https://file.lantingshucheng.com/blog/v3/static/styles/gitalk.css">


<!-- jquery -->
<script src="/lib/jquery/jquery.min.js"></script>
<script src="/lib/justified-gallery/js/jquery.justifiedGallery.min.js"></script>
<script src="/lib/fancybox/jquery.fancybox.pack.js"></script>

<script type="text/javascript" src="https://file.lantingshucheng.com/blog/v3/static/scripts/md5.js"></script>
<script type="text/javascript" src="https://file.lantingshucheng.com/blog/v3/static/scripts/gitalk.min.js"></script>

<script src="/js/main.js"></script>
<!-- search -->

<!-- Google Analytics -->

<!-- Baidu Analytics -->

    <script type="text/javascript">
        var _hmt = _hmt || [];
        (function() {
            var hm = document.createElement("script");
            hm.src = "https://hm.baidu.com/hm.js?7851ca7a86be957be994acb77639a5be";
            var s = document.getElementsByTagName("script")[0];
            s.parentNode.insertBefore(hm, s);
        })();
    </script>

<!-- Disqus Comments -->


  <script type="text/javascript">
    window.onload = function(){
      if (window.location.href.indexOf('andyliwr.github.io') > -1) {
        var gitalk = new Gitalk({
          clientID: 'ad6f70df0c70d2b715f9',
          clientSecret: '40384bee244c1e17dd7c37733d21e9ef669e969c',
          repo: 'Andyliwr.github.io',
          owner: 'Andyliwr',
          admin: ['Andyliwr'],
          id: md5(location.pathname),
          distractionFreeMode: 'true'
        })
        gitalk.render('gitalk_thread')
      }
    }
  </script>


